#!/usr/bin/python
# coding=utf-8

import os, sys, string, re, urllib

class SinaBlog:
	header = """
		<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
		<html xmlns="http://www.w3.org/1999/xhtml">
		<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head>
		<body>
	"""
	
	footer = """
		</body>
		</html>
	"""

	out_dir = '/tmp/articles'

	def __init__(self):
		self.prepare_out_dir()

	def run(self):
		pages = self.get_page_list('http://blog.sina.com.cn/s/articlelist_1770012103_5_1.html')
		
		articles = []
		for page in pages:
			print page
			articles = articles + self.get_article_list(page)

		self.get_article_content_in_single_file(articles)
		# self.get_article_content_in_multi_file(articles)

	def get_page_list(self, url):
		pages = []
		while True:
			pages.append(url)
			content = self.get_url_content(url)
			m = re.search('<li class="SG_pgnext"><a href="(.*)"\stitle', content)
			if m != None:
				url = m.group(1)
			else:
				break
		pages.reverse()
		return pages

	def get_article_list(self, url):
		s = self.get_url_content(url)

		articles = []
		start = 0
		end = 0
		while True:
			start = s.find('<span class="atc_title">', end)
			if (start >= 0) :
				end = s.find('</span>', start)
				m = re.search('href="(.*)">', s[start: end], re.DOTALL|re.MULTILINE|re.IGNORECASE)
				url = m.group(1)
				articles.append(url)
			else:
				break
		articles.reverse()
		return articles

	def get_article_content_in_multi_file(self, articles):
		for article in articles:
			index_file = self.out_dir + '/index.html'

			result = self.get_article_content(article)
			title = result[0]
			content = result[1]

			f = open(index_file, 'a')
			f.write(self.header)
			f.write('<a href="' + title + '.html">' + title + '</a><br/>')
			f.write(self.footer)
			f.close()

			f = open(self.out_dir + '/' + title + ".html", 'w')
			f.write(self.header)
			f.write(content)
			f.write(self.footer)
			f.close()

	def get_article_content_in_single_file(self, articles):
		out_file = self.out_dir + '/' + "article.html"
		if os.path.exists(out_file):
			os.remove(out_file)

		f = open(out_file, 'a')
		f.write(self.header)
		for article in articles:
			result = self.get_article_content(article)
			title = result[0]
			content = result[1]
			f.write(content)
		f.write(self.footer)
		f.close()

	def get_article_content(self, url):
		s = self.get_url_content(url)
		flags = re.DOTALL|re.MULTILINE|re.IGNORECASE
		m = re.search("<h2\s.*class=\"titName\sSG_txta\".*>(.*)</h2>", s, flags)
		title = m.group(1)
		# print title

		m = re.search('<!-- 正文开始 -->(.*)<!-- 正文结束 -->', s, flags)
		content = m.group(1)

		# pattern = re.compile("<WBR>|<BR\s*/>|&NBSP;|<P></P>|<A\s.*>.*</A>", flags)
		# pattern = re.compile("<WBR>|<BR\s*/>|&NBSP;|<P></P>", flags)
		# content = re.subn(pattern, "", content)[0]
		# print content	

		return [title, content]

	def get_url_content(self, url):
		f = urllib.urlopen(url)
		return f.read()

	def prepare_out_dir(self):
		if os.path.exists(self.out_dir) == False:
			os.makedirs(self.out_dir)

t = SinaBlog()
t.run()
